import asyncio
from pyppeteer import launch
from pyppeteer import launcher
import ssl
import random
import time
import datetime
import re
import logging
from insert_sql import insert_etl_db
import math


ssl._create_default_https_context = ssl._create_unverified_context
#launcher.AUTOMATION_ARGS.remove("--enable-automation")
#launcher.DEFAULT_ARGS.remove("--enable-automation")
miss_dict={}

class pyppeteer_crawl_xmob:

    def __init__(self, name_password_dict):

        self.name_password_dict = name_password_dict
        self.chromium_dir = "/bin/chromium-browser"
        self.login_url ='http://ad.xin-mob.com/'
        self.name_selecotor = '#username'
        self.password_selector = '#password'
        self.button_selector = '#login > button > span'

    async def login(self,semaphore,url,name,password):
        async with semaphore:
            try:
                self.browser = await launch(headless=True, executablePath=self.chromium_dir, autoClose=True, ignoreDefaultArgs=['--enable-automation'])
                self.page = await self.browser.newPage()
                await self.page.evaluateOnNewDocument('() =>{ Object.defineProperties(navigator,'
                               '{ webdriver:{ get: () => false } }) }')
                await self.page.goto(url,{'timeout': 1000*44})
                await asyncio.sleep(round(random.uniform(0.71, 1.59), 1))
                await self.page.type(self.name_selecotor, name, {'delay': random.randint(10, 30)})
                await self.page.type(self.password_selector, password, {'delay': random.randint(10, 30)})
                await asyncio.wait([self.page.waitForNavigation({'timeout': 1000*30}),self.page.click(self.button_selector)])
                await asyncio.sleep(round(random.uniform(1.71, 2.59), 1))
                print(self.page.url)
                await self.page.goto(self.page.url, {'timeout': 1000 * 44})
                # 账户
                user_html_elements = await self.page.xpath('/html/body/div/div/header/nav/div/ul[1]/li/a')
                for item in user_html_elements:
                    user = await (await item.getProperty('textContent')).jsonValue()
                await self.page.evaluate("""{window.scrollBy(0, document.body.scrollHeight);}""")
                time.sleep(round(random.uniform(0.71, 1.39), 1))
                #昨日按钮
                click_handle = await self.page.xpath('/html/body/div/div/div/section[2]/div/div/div/form/div[4]/div/button[2]/span')
                await click_handle[0].click()
                click_handle = await self.page.xpath('/html/body/div/div/div/section[2]/div/div/div/form/div[4]/div/button[2]/span')
                await click_handle[0].click()
                time.sleep(round(random.uniform(1.71, 2.39), 1))
                #yuanma = await self.page.content()
                #总条数
                count_all_html_elements = await self.page.xpath('/html/body/div/div/div/section[2]/div/div/div/div[4]/div/span[1]')
                for item in count_all_html_elements:
                    count = await (await item.getProperty('textContent')).jsonValue()
                    count_num=re.findall(r"\d+", count)[0].strip()
                # 汇总金额
                sum_jine=[]
                for i in range(1,int(count_num)+1):
                    jine_all_html_elements = await self.page.xpath('/html/body/div/div/div/section[2]/div/div/div/div[3]/div/div[3]/table/tbody/tr[%i]/td[5]/div/div/span'%i)
                    for item in jine_all_html_elements:
                        jine = await (await item.getProperty('textContent')).jsonValue()
                        print (jine)
                        sum_jine.append(jine.strip())
                print(name, '########', '账户：', user, '汇总金额：', math.ceil(sum(list(map(float, sum_jine)))*100))
                jine_sum = math.ceil(sum(list(map(float, sum_jine)))*100)
                if jine_sum:
                    insert_etl_db(str(jine_sum), 'toufang_xmob', 'jiuqi').insert_mysql()

            except Exception as e:
                print (e)
                print (datetime.datetime.now(), "!!!!!!!!xmob####\t\t",name,password)
                miss_dict[name] = password
            finally:
                #pass
                await self.browser.close()

    async def main(self):

        semaphore = asyncio.Semaphore(4)
        tasks = [
            self.login(semaphore,self.login_url,name,password) for name,password in self.name_password_dict.items()
        ]
        await asyncio.wait(tasks)

    @staticmethod
    def offical(name_password_dict,miss_dict):

        asyncio.get_event_loop().run_until_complete(pyppeteer_crawl_xmob(name_password_dict).main())
        print ("####--not success####\n",miss_dict)

        time_num = 15
        while True:
            miss_dict2 = miss_dict
            if len(miss_dict) > 0:
                time.sleep(time_num)
                miss_dict = {}
                asyncio.get_event_loop().run_until_complete(pyppeteer_crawl_xmob(miss_dict2).main())
                print ("###########################\n\t\tafter", time_num, "####--still not success####\n",miss_dict)
                time_num += 15
                if time_num == 60:
                    break
            else:
                break

def xmob():
    name_password_dict = {'email':'passwd'
                          }
    pyppeteer_crawl_xmob.offical(name_password_dict,miss_dict)

if __name__ == '__main__':

    name_password_dict = {'email':'passwd'
                          }
    pyppeteer_crawl_xmob.offical(name_password_dict,miss_dict)
    
